#coding:utf-8
"""
*   TG_Coder : Dzlua
*   Email    : 505544956@qq.com
*   Time     : 2017/08/16
"""

import threadpool
import urllib
import urllib2
import cookielib
import codecs
import re

#-----------------------
class Spider:
    def __init__(self, threads, tasks = 0):
        self.__pool = threadpool.ThreadPool(threads, tasks)
        self.__headers = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3012.0 Safari/537.36' }
        self.__cookie = cookielib.CookieJar()
        self.__handler = urllib2.HTTPCookieProcessor(self.__cookie)
        self.__opener = urllib2.build_opener(self.__handler)
    def __getPage(self, url, data):
        try:
            en_data = None
            if data:
                en_data = urllib.urlencode(data)
            request = urllib2.Request(url, en_data, self.__headers)
            response = self.__opener.open(request)
            read = response.read()
            rtype = type(read)
            print rtype
            print('-------------------------------')
            return read
            '''
            if rtype == unicode:
                return read
            elif rtype == str:
                return read.decode('utf-8')
            '''
        except urllib2.URLError, e:
            if hasattr(e,"reason"):
                err = u"[Spider] Error 连接 %s 失败,错误原因: %s" % (url, e.reason)
                self.write(err + '\r\n') 
                print(err)
                return None
    def put(self, callback, url, data = None, args = None):
        def fun(url):
            return self.__getPage(url, data)
        def funcallback(a, b):
            callback(b, a.args[0], args)
        reqs = threadpool.makeRequests(fun, [url], funcallback)  
        [ self.__pool.putRequest(req) for req in reqs ]
        print('[Spider] put url : %s, data : %s, args : %s' % (url, data, args))
    def wait(self):
        self.__pool.wait()
    def write(self, text):
        f = codecs.open(self.__class__.__name__.lower() + '.log', 'a', 'utf-8')
        f.write(text)
        f.close()
    def log(self, msg='', bfile=True):
        errmsg = '[%s] : %s' % (
            self.__class__.__name__.lower(),
            msg or '')
        if bfile:
            self.write(errmsg + '\n')
        print(errmsg)
    def url_remove_param(self, url):
        return re.sub(r'(\?[a-zA-Z0-9&=]+)?','', url)
#-----------------------
